import pandas as pd

data = pd.read_csv('data.csv')

pd.set_option('display.max_columns',None)
data.columns
data.describe()

features_mean = list(data.columns[2:12]) # 平均值数据
features_se = list(data.columns[12:22])  # 标准差数据
features_worst = list(data.columns[22:32])  # 最大值数据

data.drop('id',axis=1,inplace=True)

# 更改内容
data.diagnosis = data.diagnosis.map({'M':1,'B':0}) # map映射
data.head()

import seaborn as sns
import matplotlib.pyplot as plt
plt =sns.countplot(data.diagnosis,label='Count')#条形图

# 研究相关性
corr = data[features_mean].corr()

sns.heatmap(corr,annot=True)

features_remain = ['radius_mean','texture_mean', 'smoothness_mean','compactness_mean','symmetry_mean', 'fractal_dimension_mean']

from sklearn.model_selection import train_test_split
from sklearn import svm
from sklearn import metrics
from sklearn.preprocessing import StandardScaler

train,test = train_test_split(data,test_size=0.3)
train_X = train[features_remain]
train_y = train['diagnosis']
test_X = test[features_remain]
test_y = test['diagnosis']

ss = StandardScaler()
train_X = ss.fit_transform(train_X)
test_X = ss.fit_transform(test_X)

model = svm.SVC()
# 训练数据
model.fit(train_X,train_y)
# 6.模型评估      # 分类阶段
prediction = model.predict(test_X)
print('准确率：',metrics.accuracy_score(prediction,test_y))

%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

# use seaborn plotting defaults
import seaborn as sns
sns.set()

from sklearn.datasets import make_blobs
X, y = make_blobs(n_samples=50, centers=2,
                  random_state=0, cluster_std=0.60)
plt.scatter(X[:, 0], X[:, 1], c=y, s=50, cmap='autumn');

xfit = np.linspace(-1, 3.5)
plt.scatter(X[:, 0], X[:, 1], c=y, s=50, cmap='autumn')

for m, b in [(1, 0.65), (0.5, 1.6), (-0.2, 2.9)]:
    plt.plot(xfit, m * xfit + b, '-k')

plt.xlim(-1, 3.5);

xfit = np.linspace(-1, 3.5)
plt.scatter(X[:, 0], X[:, 1], c=y, s=50, cmap='autumn')

for m, b, d in [(1, 0.65, 0.33), (0.5, 1.6, 0.55), (-0.2, 2.9, 0.2)]:
    yfit = m * xfit + b
    plt.plot(xfit, yfit, '-k')
    plt.fill_between(xfit, yfit - d, yfit + d, edgecolor='none',
                     color='#AAAAAA', alpha=0.4)

plt.xlim(-1, 3.5);

from sklearn.svm import SVC # "Support vector classifier"
model = SVC(kernel='linear', C=1E10)
model.fit(X, y)

def plot_svc_decision_function(model, ax=None, plot_support=True):
    """Plot the decision function for a 2D SVC"""
    if ax is None:
        ax = plt.gca()
    xlim = ax.get_xlim()
    ylim = ax.get_ylim()
    
    # create grid to evaluate model
    x = np.linspace(xlim[0], xlim[1], 30)
    y = np.linspace(ylim[0], ylim[1], 30)
    Y, X = np.meshgrid(y, x)
    xy = np.vstack([X.ravel(), Y.ravel()]).T
    P = model.decision_function(xy).reshape(X.shape)
    
    # plot decision boundary and margins
    ax.contour(X, Y, P, colors='k',
               levels=[-1, 0, 1], alpha=0.5,
               linestyles=['--', '-', '--'])
    
    # plot support vectors
    if plot_support:
        ax.scatter(model.support_vectors_[:, 0],
                   model.support_vectors_[:, 1],
                   s=300, linewidth=1, facecolors='none');
    ax.set_xlim(xlim)
    ax.set_ylim(ylim)

plt.scatter(X[:, 0], X[:, 1], c=y, s=50, cmap='autumn')
plot_svc_decision_function(model);

model.support_vectors_

def plot_svm(N=10, ax=None):
    X, y = make_blobs(n_samples=200, centers=2,
                      random_state=0, cluster_std=0.60)
    X = X[:N]
    y = y[:N]
    model = SVC(kernel='linear', C=1E10)
    model.fit(X, y)
    
    ax = ax or plt.gca()
    ax.scatter(X[:, 0], X[:, 1], c=y, s=50, cmap='autumn')
    ax.set_xlim(-1, 4)
    ax.set_ylim(-1, 6)
    plot_svc_decision_function(model, ax)

fig, ax = plt.subplots(1, 2, figsize=(16, 6))
fig.subplots_adjust(left=0.0625, right=0.95, wspace=0.1)
for axi, N in zip(ax, [60, 120]):
    plot_svm(N, axi)
    axi.set_title('N = {0}'.format(N))

from sklearn.datasets import fetch_lfw_people
faces = fetch_lfw_people(min_faces_per_person=60)
print(faces.target_names)
print(faces.images.shape)

fig, ax = plt.subplots(3, 5)
for i, axi in enumerate(ax.flat):
    axi.imshow(faces.images[i], cmap='bone')
    axi.set(xticks=[], yticks=[],
            xlabel=faces.target_names[faces.target[i]])

from sklearn.svm import SVC
from sklearn.decomposition import PCA
from sklearn.pipeline import make_pipeline
pca = PCA(n_components=150, whiten=True, random_state=42)
svc = SVC(kernel='linear', class_weight='balanced')
model = make_pipeline(pca, svc)

from sklearn.model_selection import train_test_split
Xtrain, Xtest, ytrain, ytest = train_test_split(faces.data, faces.target,
                                                random_state=42)

from sklearn.model_selection import GridSearchCV

param_grid = {'svc__C': [1, 5, 10, 50]}
grid = GridSearchCV(model, param_grid)

%time grid.fit(Xtrain, ytrain)
print(grid.best_params_)

model = grid.best_estimator_
yfit = model.predict(Xtest)

fig, ax = plt.subplots(4, 6)
for i, axi in enumerate(ax.flat):
    axi.imshow(Xtest[i].reshape(62, 47), cmap='bone')
    axi.set(xticks=[], yticks=[])
    axi.set_ylabel(faces.target_names[yfit[i]].split()[-1],
                   color='black' if yfit[i] == ytest[i] else 'red')
fig.suptitle('Predicted Names; Incorrect Labels in Red', size=14);

from sklearn.metrics import classification_report
print(classification_report(ytest, yfit,
                            target_names=faces.target_names))

from sklearn.metrics import confusion_matrix
mat = confusion_matrix(ytest, yfit)
sns.heatmap(mat.T, square=True, annot=True, fmt='d', cbar=False,
            xticklabels=faces.target_names,
            yticklabels=faces.target_names)
plt.xlabel('true label')
plt.ylabel('predicted label');

from matplotlib import pyplot as plt
import jieba # 分词
import re # 正则
from sklearn.feature_extraction.text import TfidfVectorizer #tf-idf
import numpy as np

def read_data(path, is_pos=None):
    """
    给定文件的路径，读取文件
    path: path to the data
    is_pos: 是否数据是postive samples. 
    return: (list of review texts, list of labels) 
    """
    reviews, labels  = [], []
    with open(path, 'r',encoding='utf-8') as file:
        review_start  = False
        review_text = []
        for line in file:
            line = line.strip()
            if not line: continue
            if not review_start and line.startswith("<review"):
                review_start = True
                if "label" in line:
                    labels.append(int(line.split('"')[-2]))
                continue                
            if review_start and line == "</review>":
                review_start = False
                reviews.append(" ".join(review_text))
                review_text = []
                continue
            if review_start:
                review_text.append(line)
    if is_pos:
        labels = [1]*len(reviews)
    elif not is_pos is None:
        labels = [0]*len(reviews)
    return reviews, labels


def process_file():
    """
    读取训练数据和测试数据，并对它们做一些预处理
    """    
    train_pos_file = "data_sentiment/train.positive.txt" # 训练数据
    train_neg_file = "data_sentiment/train.negative.txt" # 训练数据
    test_comb_file = "data_sentiment/test.combined.txt"  # 测试数据
    
    # 读取文件部分，把具体的内容写入到变量里面
    train_pos_cmts, train_pos_lbs = read_data(train_pos_file, True)
    train_neg_cmts, train_neg_lbs = read_data(train_neg_file, False)
    train_comments = train_pos_cmts + train_neg_cmts
    train_labels = train_pos_lbs + train_neg_lbs
    test_comments, test_labels = read_data(test_comb_file)
    return train_comments, train_labels, test_comments, test_labels
train_comments, train_labels, test_comments, test_labels = process_file()

# 训练数据和测试数据大小
print (len(train_comments), len(test_comments))

print (train_comments[1], train_labels[1])

def load_stopwords(path):
    """
    从外部文件中导入停用词
    """
    stopwords = set()
    with open(path, 'r',encoding='utf-8') as in_file:
        for line in in_file:
            stopwords.add(line.strip())
    return stopwords


def clean_non_chinese_symbols(text):
    """
    处理非中文字符
    """
    text = re.sub('[!！]+', "!", text)
    text = re.sub('[?？]+', "?", text)
    text = re.sub("[a-zA-Z#$%&\'()*+,-./:;：<=>@，。★、…【】《》“”‘’[\\]^_`{|}~]+", " UNK ", text) #用unk代替特殊的字符
    return re.sub("\s+", " ", text) # 处理空格  

def clean_numbers(text):
    """
    处理数字符号  128  190  NUM 
    """
    return re.sub("\d+", ' NUM ', text) 

def preprocess_text(text, stopwords):
    """
    文本的预处理过程
    """
    text = clean_non_chinese_symbols(text)
    text = clean_numbers(text)
    text = " ".join([term for term in jieba.cut(text) if term and not term in stopwords])
    return text

path_stopwords = "./data_sentiment/stopwords.txt"
stopwords = load_stopwords(path_stopwords)

# 对于train_comments, test_comments进行字符串的处理，几个考虑的点：
#   1. 停用词过滤
#   2. 去掉特殊符号
#   3. 去掉数字（比如价格..)
#   4. ...
#   需要注意的点是，由于评论数据本身很短，如果去掉的太多，很可能字符串长度变成0
#   预处理部部分，可以自行选择合适的方案，只要注释就可以。

train_comments_new = [preprocess_text(comment, stopwords) for comment in train_comments]
test_comments_new = [preprocess_text(comment, stopwords) for comment in test_comments]

print (train_comments_new[0], test_comments_new[0])

#   利用tf-idf从文本中提取特征,写到数组里面. 
#   参考：https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html
tfidf = TfidfVectorizer()
X_train =  tfidf.fit_transform(train_comments_new) # 训练数据的特征
y_train =  train_labels # 训练数据的label
X_test = tfidf.transform(test_comments_new) # 测试数据的特征
y_test = test_labels# 测试数据的label

print (np.shape(X_train), np.shape(X_test), np.shape(y_train), np.shape(y_test))

# 8064个样本 词库共有23101

from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score

clf = MultinomialNB()
# 利用朴素贝叶斯做训练
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print("朴素贝叶斯的准确率: ", accuracy_score(y_test, y_pred))

from sklearn.neighbors import KNeighborsClassifier
clf = KNeighborsClassifier(n_neighbors=1)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print("KNN算法准确率: ", accuracy_score(y_test, y_pred))

from sklearn.linear_model import LogisticRegression

clf = LogisticRegression(solver='liblinear')
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print("逻辑回归的准确率: ", accuracy_score(y_test, y_pred))

import pandas as pd

data = pd.read_csv('spam.csv',encoding='latin')
data.head()
# ham 正常 spam垃圾邮件

data.rename(columns={"v1":"Label","v2":'Text'},inplace=True) #重命名
data.head()

# 把ham改为0 ，spam改为1
data['num_label'] =  data['Label'].map({"ham":0,"spam":1})

data.head()

print("正常邮件的数目为:{}".format(len(data[data['num_label'] == 0])))
print("垃圾邮件的数目为:{}".format(len(data[data['num_label'] == 1])))
print('总共邮件有{}'.format(len(data[data['num_label'] == 1])+len(data[data['num_label'] == 0])))

data.loc[1,'Text'] # [1.'Text'] 取第2行的数据中的Text列

# 统计文本长度
text_lengths = [len(data.loc[i,'Text']) for i in range(len(data))]
print('最短的邮件内容长度为{}'.format(min(text_lengths)))

import matplotlib.pyplot as plt

plt.hist(text_lengths,50,facecolor='blue',alpha=0.5) # bins =100 分为100个格子
plt.xlim([0,200]) # 重新设置x长度
plt.show()

# 需要把文章转换成向量的形式
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import CountVectorizer # 能把某个词语转换成向量的形式

# stopset = set(stopwords.words("english"))
# 解决停用词可选可不选（在文本中没有什么作用的词语：他（he） 她（she） 一个（an/a），这个（this），那个（that）.....）


# 构建文本向量
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(data.Text)
y = data.num_label
print(X) #稀疏矩阵

from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=2021,test_size=0.2)

print('训练数据为{}'.format(X_train.shape[0]))
print('测试数据为{}'.format(X_test.shape[0]))

# 利用朴素贝叶斯做训练
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score

clf = MultinomialNB(alpha = 1.0,fit_prior = True) # alpha =1.0 使用平滑的方法， fit_prior使用先验概率
clf.fit(X_train,y_train)
y_pred = clf.predict(X_test)
print("模型拟合的准确率为{:.1f}%".format(accuracy_score(y_test,y_pred)*100))

# 打印混淆矩阵
from sklearn.metrics import confusion_matrix
confusion_matrix(y_test,y_pred,labels=[0,1])  # 以0 和1的方式展示它，两个不同的标签，使用展示的就是2*2

from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test,y_pred,labels=[0,1]) #混淆矩阵
  
import matplotlib.pyplot as plt #导入作图库
plt.matshow(cm, cmap=plt.cm.cool) #画混淆矩阵图，配色风格使用cm.Greens，更多风格请参考官网。
plt.colorbar() #颜色标签

for x in range(len(cm)): #数据标签
    for y in range(len(cm)):
        plt.annotate(cm[x,y], xy=(x, y), horizontalalignment='center', verticalalignment='center')

plt.ylabel('True label') #坐标轴标签
plt.xlabel('Predicted label') #坐标轴标签
plt.show()

import pandas as pd
import numpy as np
from sklearn import preprocessing
import matplotlib.pyplot as plt 
plt.rc("font", size=14)
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
import seaborn as sns
sns.set(style="white")
sns.set(style="whitegrid", color_codes=True)

data = pd.read_csv('banking.csv') 
data.head()

data.info()
print(data.shape)

data.dropna(inplace=True)
print(data.columns)

data['education'].unique()

# 觉得基础教育4-9年没区别，所以放在一组
data['education'] = np.where(data['education']=='basic.4y','Basic',data['education'])
data['education'] = np.where(data['education']=='basic.6y','Basic',data['education'])
data['education'] = np.where(data['education']=='basic.9y','Basic',data['education'])
data['education'].unique()

data['y'].value_counts()

sns.countplot(x='y',data=data,palette='hls')
plt.show()
#数据特别不平衡，差别太大比例是7：1

#尽量0和1把比例控制在1：1
count_no_sub = len(data[data['y']==0])
count_sub = len(data[data['y']==1])
pct_of_no_sub = count_no_sub / (count_no_sub+count_sub)
print("未开户的百分比: %.2f%%"%(pct_of_no_sub*100))
pct_of_sub = count_sub /(count_no_sub+count_sub)
print("开户的百分比：%.2f%%"%(pct_of_sub*100))

data.groupby('y').mean()

data.groupby('job').mean()

data.groupby('marital').mean()

data.groupby('education').mean()

%matplotlib inline
table=pd.crosstab(data.job,data.y)
table.div(table.sum(1).astype(float), axis=0).plot(kind='bar', stacked=True)
plt.title('Stacked Bar Chart of Job title vs Purchase')
plt.xlabel('Job')
plt.ylabel('Proportion of Purchase')
plt.savefig('purchase_vs_job')

table=pd.crosstab(data.marital,data.y)
table.div(table.sum(1).astype(float), axis=0).plot(kind='bar', stacked=True)
plt.title('Stacked Bar Chart of Marital Status vs Purchase')
plt.xlabel('Marital Status')
plt.ylabel('Proportion of Customers')
plt.savefig('mariral_vs_pur_stack')

table=pd.crosstab(data.education,data.y)
table.div(table.sum(1).astype(float), axis=0).plot(kind='bar', stacked=True)
plt.title('Stacked Bar Chart of Education vs Purchase')
plt.xlabel('Education')
plt.ylabel('Proportion of Customers')
plt.savefig('edu_vs_pur_stack')

table=pd.crosstab(data.day_of_week,data.y)#.plot(kind='bar')
table.div(table.sum(1).astype(float), axis=0).plot(kind='bar', stacked=True)
plt.title('Stacked Bar Chart of Day of Week vs Purchase')
plt.xlabel('Day of Week')
plt.ylabel('Proportion of Purchase')
plt.savefig('dow_vs_purchase')

cat_vars=['job','marital','education','default','housing','loan','contact','month','day_of_week','poutcome']
for var in cat_vars:
    cat_list = pd.get_dummies(data[var], prefix=var)
    data=data.join(cat_list)

data_final=data.drop(cat_vars, axis=1)
data_final.columns.values

X = data_final.loc[:, data_final.columns != 'y'] #把不为y的这几列拿出来 （y为判断列）
y = data_final.loc[:, data_final.columns == 'y'].values.ravel()  #把为y的这几列拿出来

# 进行过采样
from imblearn.over_sampling import SMOTE
os = SMOTE(random_state=0) # 给一个随机数的种子
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
columns = X_train.columns
os_data_X,os_data_y=os.fit_resample(X_train, y_train)
os_data_X = pd.DataFrame(data=os_data_X,columns=columns )
os_data_y= pd.DataFrame(data=os_data_y,columns=['y'])
# we can Check the numbers of our data
print("过采样以后的数据量: ",len(os_data_X))
print("未开户的用户数量: ",len(os_data_y[os_data_y['y']==0]))
print("开户的用户数量: ",len(os_data_y[os_data_y['y']==1]))
print("未开户的用户数量的百分比: ",len(os_data_y[os_data_y['y']==0])/len(os_data_X))
print("开户的用户数量的百分比: ",len(os_data_y[os_data_y['y']==1])/len(os_data_X))

from sklearn.linear_model import LogisticRegression
from sklearn import metrics
#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
logreg = LogisticRegression()
logreg.fit(os_data_X, os_data_y.values.reshape(-1))

y_pred = logreg.predict(X_test) #测试数据
print("训练出来的准确率为:{:.2f}".format(logreg.score(X_test,y_test)))

# metrics可以得到一个报告：准确度 召回率  值越大越好  support:测试数据里面有多少这个类别的
#评价分类模型效果
from sklearn.metrics import classification_report
print(classification_report(y_test,y_pred)) # 测试数据不需要进行过采样，

from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
logit_roc_auc = roc_auc_score(y_test, logreg.predict(X_test))
fpr, tpr, thresholds = roc_curve(y_test, logreg.predict_proba(X_test)[:,1])
plt.figure()
plt.plot(fpr, tpr, label='Logistic Regression (area = %0.2f)' % logit_roc_auc)
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic')
plt.legend(loc="lower right")
plt.savefig('Log_ROC')
plt.show()

# 你可以使用如下的方法下载某一个公司的股票交易历史
# 000001 为平安银行
# 如果你还没有安装, 可以使用 pip install tushare 安装tushare python包
import tushare as ts
df = ts.get_hist_data('000001')
print(df)
df.to_csv('000001.csv')

# 预测股票
import numpy as np # 数学计算
import pandas as pd # 数据处理, 读取 CSV 文件 (e.g. pd.read_csv)
import matplotlib.pyplot as plt
from datetime import datetime as dt  #做时间处理

df = pd.read_csv('000008.csv') #平安银行股票信息
df.head()

df.shape

df.info()

df['date'].head()
# 发现日期是字符串的类型，要把它转换成时间序列

df['date'] = pd.to_datetime(df['date'])
df['date'].head()

df = df.set_index('date') #s设置时间序列为索引，不能成为预测值
df.head()

df.sort_values(by='date',inplace=True,ascending=True)  #ascending时间升序排序
df.tail()

# 检测是否有缺失值,并处理缺失值
df.dropna(axis=0,inplace=True)
df.isna().sum()

Min_date = df.index.min()
Max_date = df.index.max()
print("起始日期为",Min_date)
print("终止日期为",Max_date)
print("总共间隔时间为:",Max_date-Min_date) #包括了休息日和工作日

from plotly import tools
from plotly.graph_objs import *
from plotly.offline import init_notebook_mode, iplot, iplot_mpl
init_notebook_mode()
import chart_studio.plotly as py
import plotly.graph_objs as go

trace = go.Ohlc(x=df.index, open=df['open'], high=df['high'], low=df['low'], close=df['close'])
data = [trace]
iplot(data, filename='simple_ohlc')

from sklearn.linear_model import LinearRegression
from sklearn import preprocessing

# 创建新的列，填入预测值，根据当前的数据预测5天以后的收盘价
num = 5 #预测5天以后的情况
df['label'] = df['close'].shift(-num) #数据整体往上移动num格 也就是你这一天预测num天后的实际值
df.shape

df.tail()

# 删除‘label' 'price_change','p_change' 因为不需要他们进行预测
data = df.drop(['label','price_change','p_change'],axis=1) #删除这三列
data.tail()

# 处理特征值
X = data.values
X = preprocessing.scale(X) #标准化
# 对每个属性/每列来说所有数据都聚集在0附近，方差值为1。
X =X[:-num] #去掉后5天的值，因为要当作无后面5天值，要去预测

# 处理标签
df.dropna(inplace= True)
Target = df.label
y = Target.values

print(np.shape(X),np.shape(y))
X

##### 将数据分为训练数据和测试数据
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=2021,test_size=0.2) #设置随机数种子 设置测试集0.2
# #自己设置比例
# X_train,y_train = X[0:550,:],y[0:550]
# X_test,y_test = X[550:,:],y[550:602]
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)

lr = LinearRegression() #逻辑回归模型
lr.fit(X_train,y_train)
lr.score(X_test,y_test) #使用绝对系数 R^2 评估模型
# 绝对系数越大越拟合

# 做预测 
X_Predict = X[-num:]
Forecast = lr.predict(X_Predict)
print(Forecast) #预测最后5天的结果
print(y[-num:]) #实际最后5天的结果
print(X_Predict)  #最后5天的特征

# 预测2021-07-21 到 2021-07-27 ,一共5天的收盘价
trange = pd.date_range('2021-07-19',periods=num,freq='d')
trange

# 产生预测值dataframe
Predict_df = pd.DataFrame(Forecast, index=trange)
Predict_df.columns = ['forecast']
Predict_df

# 将预测值添加到原始dataframe
df = pd.read_csv('./000001.csv') 
df['date'] = pd.to_datetime(df['date'])
df = df.set_index('date')
# 按照时间升序排列
df.sort_values(by=['date'], inplace=True, ascending=True)
df_concat = pd.concat([df, Predict_df], axis=1)

df_concat = df_concat[df_concat.index.isin(Predict_df.index)]
df_concat.tail(num)

# Axes：坐标系 
#Subplot
fig = plt.figure(figsize=(16,9))
ax1 = fig.add_subplot(221)  #221代表子图的位置  221：几行几列 第几个  2行2列第一个

ax2 = fig.add_subplot(222)  #
ax3 = fig.add_subplot(212)
# ax4 = fig.add_subplot(224)
ax1.plot(df_concat['forecast'],linewidth=1,label='实际值',color="orange")
ax2.plot(df_concat['close'],linewidth=1,label='实际值',color="green")
ax1.set(title='预测结果',ylabel='预测值',xlabel='时间')
ax2.set(title='实际结果',ylabel='预测值',xlabel='时间')
# 画预测值和实际值

ax3 = df_concat['close'].plot(color='green', linewidth=1)
ax3 =df_concat['forecast'].plot(color='orange', linewidth=3)
plt.xlabel('时间')
plt.ylabel('价格')

plt.show()

# 理解模型
for idx, col_name in enumerate(['open', 'high', 'close', 'low', 'volume', 'ma5', 'ma10', 'ma20', 'v_ma5', 'v_ma10', 'v_ma20']):
    print("The coefficient for {} is {}".format(col_name, lr.coef_[idx]))
# 权重特征

import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns

#读取数据
df = pd.read_csv('data.csv')
df.head()  # data frame
df

df.info()

#清洗数据
# 把颜色独热编码
df_colors = df['Color'].str.get_dummies().add_prefix('Color: ') #构造独热编码
# 把类型独热编码
df_type = df['Type'].apply(str).str.get_dummies().add_prefix('Type: ') #把它当成分类
# 添加独热编码数据列
df = pd.concat([df, df_colors, df_type], axis=1)
# 去除独热编码对应的原始列
df = df.drop(['Brand', 'Type', 'Color'], axis=1) 
df.head()
# brand 都一样，不需要

df.info()

# 数据转换
matrix = df.corr()
f, ax = plt.subplots(figsize=(8, 6))
sns.heatmap(matrix, square=True)
plt.title('Car Price Variables')
 # 观察特征关联性

sns.pairplot(df[['Construction Year', 'Days Until MOT', 'Odometer', 'Ask Price']], size=2)
plt.show()

import pandas as pd
from sklearn import datasets
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA #降维
from mpl_toolkits.mplot3d import Axes3D #绘制3D图
# 主成分分析  将n维压缩成三维
pca_3 = PCA(n_components=3) #保留三组特征
# X = pca_3.fit_transform(df[['Construction Year','Odometer','Ask Price','Days Until MOT','HP','Color: black','Color: blue','Color: green','Color: grey','Color: red','Color: white','Type: 1.0','Type: 1.1','Type: 1.4']])
#PCA降维，看着使用，对比一下
X = df[['Construction Year', 'Days Until MOT', 'Odometer']] #只选3个特征。根据特征图选择
X

from sklearn.neighbors import KNeighborsRegressor
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
import numpy as np
from collections import Counter #计数器，投票器
#他是一个数，所以使用回归，不是分类
y = df['Ask Price'].values.reshape(-1, 1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1/3, random_state=41)

X_normalizer = StandardScaler() # N(0,1)  标准化
X_train = X_normalizer.fit_transform(X_train)  #依赖fit后的数据再->transform   
X_test = X_normalizer.transform(X_test)  

y_normalizer = StandardScaler()
y_train = y_normalizer.fit_transform(y_train) # 训练数据需要fit 再transform
y_test = y_normalizer.transform(y_test)

knn = KNeighborsRegressor(n_neighbors=2)  #回归，不是分类，所以可以是2
knn.fit(X_train, y_train.ravel())

#Now we can predict prices:
y_pred = knn.predict(X_test)
y_pred_inv = y_normalizer.inverse_transform(y_pred)   #转换回正常的数据
y_test_inv = y_normalizer.inverse_transform(y_test) #转换回正常的数据

# Build a plot
plt.scatter(y_pred_inv, y_test_inv)
plt.xlabel('Prediction')
plt.ylabel('Real value')

# Now add the perfect prediction line
diagonal = np.linspace(500, 1500, 100) # 500-1500 划分100个
plt.plot(diagonal, diagonal, '-r')
plt.xlabel('Predicted ask price')
plt.ylabel('Ask price')
plt.show()
# 调整画大致的预测线